import re
import logging
import traceback
import os
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
from datetime import date

from core.base_spider import BaseSpider
from config.settings import logger, BASE_DIR


class OilNewsSpider(BaseSpider):
    """
    国际原油新闻爬虫
    """
    
    def __init__(self):
        super().__init__()
        
    def _yc_driver(self, headless=True):
        """
        获取第一财经专用浏览器驱动
        """
        return self._get_driver(headless)
        
    def _save_raw_content(self, title, url, content):
        """
        保存国际原油原始内容到文件
        """
        today_date = date.today().strftime("%Y%m%d")
        raw_filename = os.path.join(BASE_DIR, "yuanshineirong", f"原始内容_{today_date}.txt")
        
        with open(raw_filename, 'a', encoding='utf-8') as raw_file:
            raw_file.write("\n\n======= 国际原油原始内容 =======\n\n")
            raw_file.write(f"标题: {title}\n")
            raw_file.write(f"链接: {url}\n")
            raw_file.write(f"内容:\n{content}\n")
        
    def crawl(self):
        """
        获取国际原油期货信息
        """
        driver = self._yc_driver(headless=True)
        try:
            logger.info("正在获取国际原油信息...")
            # 搜索国际油价
            driver.get("https://www.yicai.com/search?keys=%E5%9B%BD%E9%99%85%E6%B2%B9%E4%BB%B7")
            logger.info("正在加载搜索页面...")

            # 等待页面加载完成，使用更通用的条件
            WebDriverWait(driver, 30).until(
                lambda d: d.execute_script("return document.readyState") == "complete"
            )

            # 等待并查找新闻列表容器
            try:
                WebDriverWait(driver, 10).until(
                    EC.presence_of_element_located((By.CLASS_NAME, "f-main-news-list"))
                )
            except:
                logger.warning("未找到预期的新闻列表容器，继续处理...")

            soup = BeautifulSoup(driver.page_source, 'html.parser')

            # 查找新闻链接，使用更广泛的查找方法
            news_links = soup.find_all('a', href=True)

            article_url = None
            article_title = None
            for link in news_links:
                href = link.get('href')
                if href and '/news/' in href and href.endswith('.html'):
                    article_url = href if href.startswith('http') else "https://www.yicai.com" + href
                    article_title = link.get_text(strip=True)
                    logger.info(f"找到文章链接: {article_url}")
                    break

            if not article_url:
                logger.warning("未找到任何文章链接")
                return None

            # 访问文章页面提取原油信息
            logger.info(f"正在访问文章页面: {article_url}")
            driver.get(article_url)

            # 等待页面加载完成
            WebDriverWait(driver, 30).until(
                lambda d: d.execute_script("return document.readyState") == "complete"
            )

            # 尝试等待文章内容加载
            try:
                WebDriverWait(driver, 10).until(
                    EC.presence_of_element_located((By.ID, "multi-text"))
                )
            except:
                logger.warning("未找到 multi-text 容器，继续处理...")

            soup = BeautifulSoup(driver.page_source, "html.parser")

            # 尝试多种方式查找文章内容
            content_div = soup.select_one('div#multi-text') or soup.find('div', class_='m-txt') or soup.body

            if not content_div:
                logger.warning("未找到正文容器")
                return None

            # 提取文章全文内容
            paragraphs = content_div.find_all(['p', 'div'])
            article_content = ""
            for p in paragraphs:
                text = p.get_text(strip=True)
                if len(text) > 0:
                    article_content += text + "\n"

            if article_content:
                self._save_raw_content(article_title, article_url, article_content)
                logger.info("国际原油原始内容已保存")
                return article_content
            else:
                logger.warning("未找到文章内容")
                return None

        except Exception as e:
            logger.error(f"获取原油信息时出错: {str(e)}")
            logger.error(traceback.format_exc())
            return None
        finally:
            driver.quit()